#include "StdAfx.h"
#include "Stopword.h"
#include <fstream>
#include <iostream>
#include <sstream>
#include <set>
#include <vector>
using namespace std;


Stopword::Stopword(void)
{
	stopwordSet;
}

Stopword::~Stopword(void)
{
}

void Stopword::addStopword(string word)
{
	stopwordSet.insert(word);
}

void Stopword::removeStopword(string word)
{
	stopwordSet.erase(word);
}

set<string>::iterator Stopword::findStopword(string word)
{
	return stopwordSet.find(word);
}

set<string> Stopword::loadStopword(string stopwordfile)
{
	ifstream ifs(stopwordfile); // open file
	if(!ifs)
	{
		cerr << "Fail to open the file" << stopwordfile << endl;
	}

	string str;
	while (ifs >> str)
	{
		stopwordSet.insert(str); //push all stopwords into set
	}
	return stopwordSet;
}

/*
*	This function receives a string from caller, 
*	filter the stopwords in the string according to the list in stopwordfile, 
*	then update and return the tokenmap
*/
map<string, double>  & Stopword::filter(map<string, double> & tokenmap, string str, string stopwordfile)
{
	// load the stopwordlist file into set
	stopwordSet = loadStopword(stopwordfile);

	string buf; // have a buffer string
    stringstream ss(str); // insert the passed in string into a stream
	
	// filter the stopwords then insert the rest words into tokenmap
	while (ss >> buf)
    {    
		set<string>::iterator its = findStopword(buf); // search if buf in the stopword list
		if (its == stopwordSet.end()) // if not a stopword, insert it into map
		{
			if(tokenmap.size() == 0) // if map is empty
				tokenmap.insert(map<string, double>::value_type(buf,1)); 
				// store the word with frequency 1
			else // if map is not empty
			{
				map<string, double>::iterator itm = tokenmap.find(buf);
				if(itm == tokenmap.end()) //if no duplicate record
					tokenmap.insert(map<string, double>::value_type(buf,1));
				else //if duplicate record exist
					itm->second++;
			}
		}
	}
		
	return tokenmap ;
}